%% -------------------------------------------------------------------
%% -------------------------------------------------------------------
%% mmgraph.tex
%%
%% Created 2012-06-24 by Sohaib Ghani, BC Kwon, Seungyoon Lee,
%% Ji Soo Yi, and Niklas Elmqvist
%%
%% Intended for submission to IEEE InfoVis 2012. (rejected)
%% Revised Summer 2012 for IEEE TVCG. (rejected)
%% Revised Spring 2013 for IEEE VAST 2013. 
%% -------------------------------------------------------------------

\RequirePackage{ifpdf} % to prevent the "\ifpdf already defined" error

%% -------------------------------------------------------------------
%% Document class
%% -------------------------------------------------------------------
%\documentclass{vgtc}                          % final (conference style)
\documentclass[journal]{vgtc}                % final (journal style)
%\documentclass[review]{vgtc}                 % review
%\documentclass[review,journal]{vgtc}         % review (journal style)
%\documentclass[widereview]{vgtc}             % wide-spaced review
%\documentclass[preprint]{vgtc}               % preprint
%\documentclass[preprint,journal]{vgtc}       % preprint (journal style)
%\documentclass[electronic]{vgtc}             % electronic version
%\documentclass[electronic,journal]{vgtc}     % electronic version, journal

%% -------------------------------------------------------------------
%% Packages
%% -------------------------------------------------------------------

%% These three lines bring in essential packages: ``mathptmx'' for Type 1 
%% typefaces, ``graphicx'' for inclusion of EPS figures. and ``times''
%% for proper handling of the times font family.
%% We encourage the use of mathptmx for consistent usage of times font
%% throughout the proceedings. However, if you encounter conflicts
%% with other math-related packages, you may want to disable it.
\usepackage{mathptmx}
\usepackage{graphicx}
\usepackage{times}

% For internet addresses
\usepackage{url}

%% Subfigure support
\usepackage{subfigure}

%% For esoteric symbols
\usepackage{amsfonts}

%% Allow for this line if you want the electronic option to work properly
\vgtcinsertpkg

%% ---------------------------------------------------------------------
%% Paper metainformation
%% ---------------------------------------------------------------------

%% If you are submitting a paper to a conference for review with a double
%% blind reviewing process, please replace the value ``0'' below with your
%% OnlineID. Otherwise, you may safely leave it at ``0''.
\onlineid{0}

%% declare the category of your paper, only shown in review mode
\vgtccategory{Research}

%% -------------------------------------------------------------------
%% Title, author(s), and affiliations
%% -------------------------------------------------------------------

% Paper title
\title{Visual Analytics for Multimodal Social Network Analysis: \\ 
  A Design Study with Social Scientists}

% Authors
\author{Sohaib~Ghani,~\textit{Student member,~IEEE},
  Bum~Chul~Kwon,~\textit{Student member,~IEEE},\\
  Seungyoon Lee,
  ~Ji~Soo~Yi,~\textit{Member,~IEEE},
  and~Niklas~Elmqvist,~\textit{Senior member,~IEEE}%
}

%% Author information
\authorfooter{
\item S.\ Ghani and N.\ Elmqvist are with the School of Electrical and
  Computer Engineering, Purdue University, West Lafayette, IN, USA.
  E-mail: \{sghani, elm\}@purdue.edu
\item B. C. Kwon and J. S. Yi are with the School of Industrial
  Engineering, Purdue University, West Lafayette, IN, USA.  E-mail:
  \{kwonb, yij\}@purdue.edu
\item S. Lee is with the Brian Lamb School of Communication, Purdue
  University, West Lafayette, IN, USA.  E-mail: seungyoon@purdue.edu
}

%% Short author title
\shortauthortitle{Parallel Node-Link Bands}

%% Manuscript note
\manuscriptnote{Submitted to IEEE VAST 2013.  Do not redistribute.}

%% In preprint mode you may define your own headline.
\preprinttext{Submitted to IEEE VAST 2013.  Do not redistribute.}

% Teaser figure goes here
\teaser{
  \centering
  \vspace{0.25cm}
  \resizebox{0.9\textwidth}{!}{\includegraphics{figures/va_sna_comm_cropped}}
  \caption{Four design sketches for visualizing multimodal social
    networks, evolved through discussion between social network
    experts and visual analytics experts.
    Early designs (top two) focus on splitting node-link diagrams into
    separate spaces, whereas the latter (bottom left) use vertical
    bands while maintaining compatibility with node-link diagrams
    (bottom right).}
  \label{fig:sketches}
}

%% -------------------------------------------------------------------
%% Abstract
%% -------------------------------------------------------------------

\abstract{Social network analysis (SNA) is becoming increasingly
  concerned not only with actors and their relations, but also with
  distinguishing between different types of such entities.
  For example, social scientists may want to investigate asymmetric
  relations in organizations with strict chains of command, or
  incorporate non-actors such as conferences and projects when
  analyzing co-authorship patterns.
  Multimodal social networks are those where actors and relations
  belong to different types, or \textit{modes}, and multimodal social
  network analysis (mSNA) is accordingly SNA for such networks.
  In this paper, we present a design study that
  we conducted with several social scientist collaborators on how to
  support mSNA using visual analytics
  tools.
  Based on an open-ended, formative design process, we devised a
  visual representation called \textit{parallel node-link bands} (PNLBs) that
  splits modes into separate bands and renders connections between
  adjacent ones, similar to the list view in Jigsaw.
  We then used the tool in a qualitative evaluation involving five
  social scientists whose feedback informed a second design phase that
  incorporated additional network metrics.
  Finally, we conducted a second qualitative evaluation with our
  social scientist collaborators that provided further insights on the
  utility of the PNLBs representation and the potential of visual
  analytics for mSNA.}

%% Keywords that describe your work.  Will show as 'Index Terms' in journal
%% please capitalize first letter and insert punctuation after last keyword
\keywords{Design study, user-centered design, node-link diagrams,
  multimodal graphs, interaction, qualitative evaluation.}

%% Note: ACM CCS keywords not used in VGTC journal version.

%% ---------------------------------------------------------------------
%% Document Beginning
%% ---------------------------------------------------------------------
\begin{document}

%% The ``\maketitle'' command must be the first command after the
%% ``\begin{document}'' command.  It prepares and prints the title block.

%% the only exception to this rule is the \firstsection command
\firstsection{Introduction}

\maketitle

%% -------------------------------------------------------------------
%% Introduction
%% -------------------------------------------------------------------

Social network analysis (SNA)~\cite{Scott1991} is the collective name
for a family of methods used to analyze sets of social actors
connected by relations.
SNA has become increasingly important due to modern information
technologies that allow humans to connect and relate in entirely new
and easily observable ways.
As a case in point, social media websites, such as Facebook, Twitter,
and LinkedIn, include hundreds of millions of users with various types
of relations between them.
The scale and complexity of these massive networks put increasing
demands on software support for computation, statistics, and decision making.
Visual analytics is increasingly being used for this purpose, and
several new systems have been proposed that merge visual
representations and network statistics to aid social network analysis,
including Gephi~\cite{Bastian2009}, NodeXL~\cite{Hansen2010,
  Smith2009}, GraphDice~\cite{Bezerianos2010},
TimeMatrix~\cite{Yi2010}, and GUESS~\cite{Adar2006}.

However, in many real-world settings, the networks consist of not one
but several different types, or \textit{modes}, of nodes.
Examples include co-authorship networks that contain not just authors, but also the venues they attend and the
journals they publish in; organizational charts that contain employees
as well as the departments they belong to; and information retrieval
processes that involve both databases and the people who access them.
Consequently, these \textit{multimodal} (cf.~unimodal) social networks
also have multiple types of edges depending on whether the edge is
connecting nodes of the same mode (within-mode, such as representing
friendship among people) or different modes (between-mode, such as
employee affiliations with departments, or employees accessing
databases).
However, while unimodal network visualization is prevalent, as
evidenced by the examples above, few techniques exist for visualizing
multimodal graphs.
Further, in the field of social science, a standard framework for
analyzing multimodal social networks, especially those that involve
more than two modes, has not been established yet.

In this paper, we present a design study on the use of visual
analytics to aid social scientists in conducting multimodal social
network analysis (mSNA).
Retrospectively,\footnote{Sedlmair et al.~\cite{Sedlmair2012} was
  published after this design study started.} we largely follow the
methodological framework proposed by Sedlmair et
al.~\cite{Sedlmair2012}.
As with any design study, a major obstacle is often to articulate user
(social scientists in our case) needs and requirements (\emph{low task
  clarity}~\cite{Sedlmair2012}).
To aid in this process, we included one of our social scientist
collaborators as an active participant in the project and a co-author
of this paper from its very beginning.
Nevertheless, this low task clarity was exacerbated by the fact that
multimodal social networks are not yet a well-established concept even
in social science, so our collaborators had a difficult
time defining requirements and features.
We therefore decided to conduct an iterative design process that
included (1) an initial open-ended formative phase, (2) a
visualization design phase, (3) an interim qualitative evaluation with
five social scientists, (4) another design iteration incorporating
additional analytics support, and (5) a second qualitative evaluation
with our domain experts validating our changes.
Due to the low task clarity of mSNA, our visual
analytics tool---\textsc{MMGraph}---played an interesting role
throughout this process: it served not only as a tool for answering
questions that our domain experts had about their data, but it also
became a living prototype that helped them see the potential in visual
analytics and aided them in coming up with new requests for future
iterations of the tool.

Our contributions in this paper include the following: (i)
characterizing the problems and abstracting some potential tasks for
multimodal social network analysis (mSNA); (ii) proposing a visual
analytics tool for mSNA (\textsc{MMGraph}) refined through multiple
iterations of design and evaluation with social scientists; and (iii)
sharing several lessons learned through this iterative design process
that are specific to the domain of social science and social network
analysis.
We want to emphasize that this research project is a design study, and
thus clearly \emph{problem-driven} and not \emph{technique-driven}. 
Thus, even though we ended up deriving some unique representation and
interaction techniques (e.g., the open-sesame interaction in
Section~\ref{sec:design-space}), demonstrating the novelty of these
techniques is \emph{not} the emphasis of this paper.

%% -------------------------------------------------------------------
%% Background
%% -------------------------------------------------------------------
\section{Background}

Social networks are universal to the human condition.
They can be used to model our relations to friends and relatives;
our connections to groups and organizations; and the social structure
of our very own neighborhood, state, or nation.
Researchers have long studied these types of social networks using
network theory and techniques through what is collectively known as
\textit{social network analysis} (SNA)~\cite{Knoke2008}.
These techniques enable the examination of social phenomena such as
whether friendship leads to the contagion of obesity (or vice versa),
what causes radicalization and terrorist group formation in extremist
communities, and the nature of user-to-user interactions on social
media platforms.
Below we review cross-disciplinary research within both social science
and visualization that this work touches.

\subsection{Definitions}
\label{sec:defs}

We define a \textit{multimodal graph} $G$ as the traditional ordered
pair $G = (V, E)$ comprised of a set of vertices $V$ and edges $E$,
but where vertices can be partitioned using a modality equivalence
relation $\sim_{mod}$.
This modality relation $\sim_{mod}$ is defined using the
notion of \textit{vertex type}, and the equivalence classes
(partitions) defined by relation are called \textit{modes}.
We can further define a modality relation for edges based on a tuple
of the modes of the two vertices an edge connects.

For example, consider the simple food chain network in
Figure~\ref{fig:food-chain}, where $V$ is the set of all entities in
the network, and $E$ are their linkages.
We can define the equivalence classes (modes), such as ``plants''
(green), ``prey animals'' (red), ``predators'' (blue), and ``habitats''
(yellow), and define the modality relation $\sim_{mod}$ to organize
vertices based on which category they belong.

\begin{figure}[tb]
  \centering
  \resizebox{0.8\columnwidth}{!}{\includegraphics{figures/food-chain}}
  \caption{Biological food chain with different node types (links
    between nodes have been omitted).}
  \label{fig:food-chain}
\end{figure}

While multimodal graphs are prevalent in many domains, this paper is
primarily motivated by SNA~\cite{Scott1991}.
We define \textit{multimodal social network analysis} (mSNA) as SNA in
the presence of \textit{multimodal social networks}, i.e., where the
social actors can be partitioned into modes (not all of them
necessarily sentient or even living).
Given this definition, the food chain in Figure~\ref{fig:food-chain}
can be regarded as a multimodal social network (even if ``biological
network'' is perhaps closer at hand).

This particular example represents one type of multimodal network where different modes represent the attributes of a single type of entity (e.g., species). The relationships represent species eating other species. Another major source of multimodal networks is multivariate tabular data, where there are multiple types of entities that have relationships with each other, as well as attributes of those entities (***Liu et al., 2012). As the modes represent different types of entities, the types of links also vary across different pairs of modes. For instance, in the dataset we work with in the current paper, PIs ``conduct" projects, program managers ``award" projects, and PIs ``belong to" institutions.

\subsection{Social Network Analysis}
\label{sec:sna}

With the advent of communication technologies (e.g., social media
websites and collaborative knowledge tools) and large computational
platforms (e.g., sensor networks and data collection tools), a massive
amount of social network data is being collected today.
Much effort has been made to analyze such data, leading to several
interesting applications of network theories to social phenomena
(e.g.,~\cite{Guimera2005, Lazer2009}).

However, to the best of our knowledge, investigation of social
networks has largely dealt with unimodal social networks (e.g.,
friendship relations among friends and co-authorship relations among
authors) and there has been an increasing need to expand our
formulation to address relations between multiple types of
nodes~\cite{Contractor2011}.
Such expansion enables a wide variety of questions to be examined
beyond those from the realm of unimodal networks.
For example, dynamics involving friendship can be better
explained by considering the way people participate in events
or affiliate with social organizations.
Co-authorship among authors can be better understood when
looking at the projects they work on as well as the
institutions they are employed by.
These additional types of nodes may have substantial effects on
the formation and dissolution of social ties, and therefore allow us to
investigate more complex social interaction among multiple types of entities.

The simplest configuration of multimodal networks is \textit{two-mode
  networks} that consist of two modes---also called affiliation
networks in the context of groups and members~\cite{Borgatti1997,
  Wasserman1994}.
A large part of existing literature on two-mode networks has dealt
with bipartite graphs, defined as a graph with nodes in two distinct
sets, and links only between nodes of one set and nodes of the
other~\cite{Latapy2008}.
These links can be considered as between-mode ties (as opposed to
within-mode ties).

A review of the relevant literature (e.g.,~\cite{Beauguitte2010,
  Borgatti1997, Faust2005}) suggests several representative measures
for multimodal networks:

\vspace{-.25cm}
\begin{enumerate}
  \setlength{\itemsep}{-1mm}
\item[Q1] Measures at the individual node level:
  
  \vspace{-.25cm}
  \begin{enumerate}
  
  \item[Q1a] Centrality: Which nodes are central in terms of
    between-mode ties? (e.g., degree centrality: which nodes in mode A
    are tied to the most nodes in mode B?; betweenness centrality:
    which nodes in mode A are most central in terms of bridging
    otherwise disconnected nodes in mode B?)

  \item[Q1b] Positions and roles: Which nodes occupy similar positions and
    roles? (e.g., structural equivalence: which nodes in mode A are
    similar in terms of their ties to nodes in B?)

  \item[Q1c] Attributes: Which nodal attributes impact between-mode
    ties?  (e.g., how does the attribute of nodes in mode A impact
    their ties to nodes in mode B?)

  \end{enumerate}
  \vspace{-.25cm}

\item[Q2] Measures at the global network level:

  \vspace{-.25cm}
  \begin{enumerate}

  \item[Q2a] Density: How dense is the network, between and within
    modes?  (What is the number of between-mode and within-mode
    links divided by the maximum number of possible links
    between-mode and within-mode, respectively?)

  \item[Q2b] Centralization: How centralized is the network in
    between-mode ties?
    (Are nodes in one mode similar in terms of number of ties they
    have to another mode, or are ties unequally distributed with a few
    dominating nodes?)
    
  \item[Q2c] Subgroups: Are there visible substructures in the
    network? (Which nodes in mode A can be clustered based on being
    connected to the same or similar nodes in mode B?)

  \end{enumerate}
  \vspace{-.25cm}

\end{enumerate}
\vspace{-.25cm}
The interpretation of these measures will vary depending on the source of multimodal data. In particular, when multimodal networks are derived from tabular data, the nodes are of different levels or entities, and the types of links will vary depending on which modes are being examined, therefore requiring a context-specific application of the measures.

So far, a standard approach to analyzing multimodal networks has been
to transform them into unimodal social networks either through
projection or through separation.
For example, the ties between manufacturers and users of a product are
transformed (or projected) to ties among manufacturers established in
case of common users~\cite{Cantner2011}.
In other cases, a manufacturer's network and a user's network are
divided and analyzed separately, or combined into a unimodal network
with the modes treated as node attributes.
While these conversions are done for simplicity as well as for
utilizing existing unimodal social network analysis (uSNA) tools, a
rich set of information is lost during this process~\cite{Latapy2008}.
First of all, networks produced through this transformation do not
represent a direct relation between actors of different modes, but an
indirect relation induced by their common affiliation to a set of
events~\cite{Beauguitte2010}.
Second, by removing one set of nodes from the data or combining the
nodes, attribute information associated with each of the modes cannot
be simultaneously considered~\cite{Singh2007}.

Given these limitations, we believe that a social network analysis
tool that presents the subtleties and complexities of multimodal
social networks can help social scientists gain useful new insights
about what they would like to learn from such multimodal networks.

\subsection{Multimodal Network Visualization}

Our review of graph visualization~\cite{Freeman2000, Freeman2004,
  Herman2000} has identified only a few studies on multimodal
networks.
We summarize these below:

\subsubsection{Compound Network Visualization}
\label{ssub:compound_network_visisualizaiton}
A common approach for visualizing a multimodal graph is to treat it as
a unimodal graph, with different colors or shapes distinguishing
between types of modes and links.
Such \textit{compound network visualizations} are found in several
visualization systems (e.g.,~\cite{Bastian2009, Borgatti2002,
  Singh2007}).
However, this approach confounds all modes within the same
view, and the resulting visual complexity can be high.
Many techniques have been studied to overcome such visual complexity from large graph~\cite{vonLandesberger_2011}.
Despite such techniques, only certain ties or nodes can be shown at any
point in time, but this naturally results in data being omitted.

\subsubsection{Eliminating Modes}

Another approach eliminates modes by
\textit{projecting}~\cite{Liu2011} nodes based on connections to a
particular mode.
This retains the connectivity structure, yet reduces the number of
nodes (by mode).
For example, if authors A and B write paper X together, the ties (A-X)
and (B-X) can be merged (\textit{projected}) into (A-B).
While this approach can lower the overall complexity, it comes at the
cost of information loss.
For example, (A-W), (B-W), and (C-W) will have the same merged network
as (A-X), (B-X), (B-Y), (C-Y), (C-Z), and (A-Z) even though in the
former case, three authors collectively wrote a single paper, while in
the latter case, three pairs of authors wrote three different papers.

\subsubsection{Linked Network Visualization}

A third approach is to use multiple views, each of which renders a
different mode of the graph separately (\textit{linked network
  visualization}).
Between-mode ties are visualized using visual links or brushing
(when nodes are selected in one view, corresponding nodes in
another view are highlighted).
VisLink~\cite{Collins2007}, semantic substrates~\cite{Aris2007,
Shneiderman2006}, the list view in Jigsaw~\cite{Stasko2008}, and
SmallWorlds~\cite{Gretarsson2010} are examples of this idea.

More specifically, all four of these examples provide distinct planes
that can be used for different mode networks, and then show
connections between the planes using graphical links.
However, while VisLink views are often less cluttered than compound
network visualizations, the view can still be visually complex due to
overlapping between-mode and within-mode ties.
When brushing is used, the clutter is reduced.
However, only a partial set of between-mode ties can be shown in this
case.
Semantic substrates, the list view in Jigsaw, and SmallWorlds are less
cluttered, but this is mainly because they do not show within-mode
ties.
When there are clear hierarchical structures between the
nodes, TreeNetViz~\cite{Gou2011}, which uses a radial space-filling
visual design, is another effective method.
However, not all multi-modal networks have such an existing
hierarchical structure.
Finally, the recent GraphTrail~\cite{Dunne2012} tool visualizes and
aggregates attributes associated with nodes and edges.
The multimodal networks are called \textit{heterogeneous networks} in
their work, but involve different types of nodes and edges.
However, their focus is on building visual queries using these graphs,
not on general visual analytics for mSNA.

\subsection{Reducing Data and Visual Complexity}
\label{sec:dimensionality}

The problem of visualizing multimodal social network is similar to ``the
curse of dimensionality''~\cite{Bellman1961}, which is a problem
occurring when multidimensional (not multimodal) data are projected onto
a 2D or 3D display.
Such projection easily generates clutter, distortion, and ensuing
confusion.
To make matters worse, visualizing a unimodal social network often
consumes two spatial dimensions while visualizing a uni-dimension data
often consumes only one spatial dimension.
Thus, it is not possible to directly borrow ideas from multidimensional
visualization to address the challenges of multimodal network
visualization.
However, we can draw upon lessons learned from previous work for
overcoming the curse of dimensionality.
A review reveals the following strategies:

\paragraph{Divide and Conquer.}

Based on subdividing a problem into smaller components until each
component is small enough to easily solve, this is one of the core
strategies in many sub-disciplines of computer science, and the same
principle has been applied to visualize multidimensional data.
However, divide and conquer only visualizes a subset of data at a
time, making the global view difficult to understand.
Examples include scatterplot matrices (SPLOMs)~\cite{Hand2001}, graph
exploration with degree-of-interest~\cite{Ham2009a}, and Worlds within
Worlds~\cite{Feiner1990}.

\paragraph{Distortion.}

In order to show the overall picture more effectively, some techniques
distort the orthogonal relationships between dimensions, thereby
gaining compactness by sacrificing familiarity.
Examples include parallel coordinates~\cite{Inselberg1990}, star
coordinates~\cite{Kandogan2001}, and Flexible LINked Axes
(FLINA)~\cite{Claessen2011}.

\paragraph{Compression.}

When the amount of data and the number of dimensions surpass a certain
level, the information may be drastically compressed using
meta-dimensional information to show the overview at the cost of
information loss.
Examples include Principal Component Analysis~\cite{Hand2001},
multidimensional scaling~\cite{Cox2001}, and
Scagnostics~\cite{Wilkinson2005}.

\paragraph{Metaphor.}

When compression and distortion cause a visualization to become
difficult to understand, some metaphors that are readily detectable
(e.g., human faces) or understandable (e.g., a magnet metaphor) can
help users deal with the complexity.
Examples include Chernoff faces~\cite{Chernoff1973} and Dust \&
Magnet~\cite{Yi2005}.

One interesting pattern common to these strategies is the
trade-off between different elements of the visualization.
If one wants to show more data or attributes either through distortion
or compression strategies, the resulting visualization becomes
visually complex.
If one would like to lower complexity through the divide and conquer
strategy, the amount of data shown in a single view will decrease.
The key issue is striking a balance between these two factors.

%% ---------------------------------------------------------------------
%% CASE STUDY: OVERVIEW
%% ---------------------------------------------------------------------
\section{Overview: Visual Analytics for mSNA}

The goal of this study was to support our social scientist colleagues
in performing multimodal social network analysis.
For this purpose, we recruited a social scientist with professional
interests in mSNA as an active collaborator and co-author for this
project (the third co-author A3).
While existing literature presents theoretical and analytical
frameworks for two-mode networks, those are yet to be fully expanded
to multimodal ones.
Therefore, we decided to employ an exploratory and user-driven
design process with the following stages:

\begin{figure*}[htb]
  \centering
  \resizebox{\textwidth}{!}{\includegraphics{figures/pnlb-view-new.png}}
  \caption{Parallel node-link bands (PNLBs) being used to visualize
    multimodal NSF funding data consisting of Institutions, PIs (and
    Co-PIs), Projects, program managers (Pr-Man), NSF programs (Programs), and NSF directorates (Dir).
    The color inside each node represent degree centrality value for
    that node.}
  \label{fig:pnlbs-example}
\end{figure*}

\vspace{-.25cm}
\begin{enumerate}
\setlength{\itemsep}{0px}
\item[I.]\textbf{Early design:} formative sketching, brainstorming,
  prototyping, and requirements gathering;
\item[II.]\textbf{Iterative tool development:} progressively refining our
  visual analytics tool based on domain expert feedback;
\item[III.]\textbf{Formative evaluation:} qualitative evaluation of our
  visual analytics tool with five social scientists;
\item[IV.]\textbf{Iterative tool refinement:} creating additional features
  based on feedback from the formative evaluation; and
\item[V.]\textbf{Summative evaluation:} qualitative evaluation of the
  current state of our visual analytics tool using domain experts.
\end{enumerate}
\vspace{-.25cm}

%% ---------------------------------------------------------------------
%% PHASE I: EARLY DESIGN
%% ---------------------------------------------------------------------
\section{Phase I: Early Design}

The early design process consisted of brainstorming, sketching
(Figure~\ref{fig:sketches}), and reviewing existing work in the domain.
We built an early prototype with a sample data set (NSF funding award
data), so that A3 could make sense of the effectiveness of visual
analytics.
This helped us come up with specific and contextualized questions that
social scientists might want to answer using the tool.
Some of these questions fall within the range of research questions
discussed in Section~\ref{sec:sna}; some do not (e.g., q5), yet add
utility to the tool:

\vspace{-.25cm}
\begin{itemize}
  \setlength{\itemsep}{0px}
\item[q1] Who are the most successful investigators?  Which
  institutions are they from?  (Q1a and Q1c)

\item[q2] Who are the collaborators of a particular investigator?
  Have they collaborated on multiple projects?  (Q1b and Q2c)

\item[q3] What are the overall patterns and trends in collaboration
  and funding? (Q2a, Q2b, and Q2c)

\item[q4] Which program manager awarded most projects? (Q1a)

\item[q5] How can I find programs with specific subjects or contents?

\end{itemize}
\vspace{-.25cm}

These questions helped prioritize features to be implemented and
worked as test cases to verify whether features have a true purpose
in designing the initial prototype.
However, at the same time, these tasks tended to be mere extensions
of tasks existing for unimodal social network analysis.
We found that it was quite challenging for our domain experts to come
up with such questions this early in the design process.
These difficulties are further discussed in
Section~\ref{sec:difficulties_of_socialscientists}.

%% ---------------------------------------------------------------------
%% PHASE II: ITERATIVE MMGRAPH TOOL DEVELOPMENT
%% ---------------------------------------------------------------------
\section{Phase II: Iterative Tool Development (MMGraph)}

Starting from our low-fidelity prototype from the early design phase
(above), we then built an initial visual analytics tool for mSNA that
we call \textsc{MMGraph}.
MMGraph is a Java application built using the Piccolo
library~\cite{Bederson2004}.
The tool loads multimodal graphs using the GraphML format where a
specific node attribute \texttt{type} is used to convey the mode of
each vertex.
The initial visualization we provided was a standard compound network
visualization (see Section~\ref{ssub:compound_network_visisualizaiton})
with color-coding to visualize mode (Figure~\ref{fig:nl-view}).

\begin{figure}[htb]
  \centering
  \resizebox{\columnwidth}{!}{\includegraphics{figures/nl-view}}
  \caption{The compound network visualization in MMGraph.
    This view is designed to be shown in parallel with the PNLBs view
    (Figure~\ref{fig:pnlbs-example}) on a second monitor.}
  \label{fig:nl-view}
\end{figure}

% \begin{figure}[htb]
%   \centering
%   \resizebox{0.8\columnwidth}{!}{\includegraphics{figures/food-chain-pnlb}}
%   \caption{Illustration of the parallel node-link band visualization
%     technique where nodes have been organized into parallel bands
%     based on mode.}
%   \label{fig:food-chain-pnlb}
% \end{figure}

\subsection{Parallel Node-Link Bands}

One of our earliest findings from the iterative design process was that
the compound network visualization was not appropriate for multimodal
social network analysis.
While it was familiar to A3, it also caused high visual complexity and
made distinguishing between within-mode and between-mode ties
difficult.
Our conclusion was that a more structured organization of the visual
space was necessary.

Based on our design process (above), we therefore added\footnote{This
means that MMGraph has both the compound network visualization and
PNLBs, which are shown in parallel on dual displays.} \textit{parallel
node-link bands} (PNLBs) (Figure~\ref{fig:pnlbs-example}) to MMGraph.
PNLBs use visual node and link marks partitioned into separate
\textit{bands} based on modes to minimize visual clutter, yet which
arranges the bands in parallel to maintain cohesiveness.
This design actively omits all edges except for those between bands
that have been placed adjacent to each other on the visual space
(neighboring between-mode network), which improves scalability without
affecting data accuracy.
In other words, all the edges between nodes within a band (within-mode
network) and all the edges between nodes that belong to
non-neighboring bands (non-neighboring between-mode network) are
hidden.
The bands can be reordered to expose other between-mode networks.
The technique draws inspiration from the list view in the investigative 
analytics tool Jigsaw~\cite{Stasko2008}, where entities of a particular 
type are arranged in separate lists and relations between entities are 
drawn as lines connecting them.

The motivation behind this visual design is to impose structure on the
multimodal graph without sacrificing the familiar node-link
representation while organizing relations between entities in a
logical manner.
Based on our discussion on strategies for dimensionality reduction
(Section~\ref{sec:dimensionality}), our approach uses a combination of
divide and conquer (not showing both within-mode and between-mode
network simultaneously) and distortion (change the node-link layout to
the parallel node-like layout) to achieve an efficient visual
representation.

Given that one of our central design requirements is to maintain
familiarity for social scientists, it is worth taking a step back to
evaluate how PNLBs Correlate to standard node-link diagrams.
One important advantage in this respect is that PNLBs retain the same
node-link diagram format, even if the layout is
different and no longer as free-form and organic.
On the other hand, this layout still follows the same basic idea as
the conventional visual layout of bipartite graphs
familiar to most social scientists.
The remaining hurdle is how to communicate the fact that PNLBs hide
all edges except for those between adjacent bands as well as those
within the same band.

\subsection{Design Space}
\label{sec:design-space}

We now explore the free parameters in the PNLBs design space to fully
map out the utility of the technique.

\paragraph{Node Representation.}

The node representations in each band should remain similar to those
in a node-link diagram, but can be augmented with additional visual
variables.
We use the border color for each band to encode mode and also add the
name of the mode to the header part of the band.
In addition, for multimodal graphs where textual data has been
associated with some of the nodes, we provided a word cloud popup
interaction that shows a frequency-based summary of the keywords
associated with the node under the mouse cursor
(Figure~\ref{fig:wordcloud}).
We use a simple tf-idf~\cite{Jones1972} mechanism to extract these
keywords.
A related approach is to provide an ego-network popup that instead
shows the direct neighbors of the node under the mouse cursor as a
node-link diagram, with the node itself at the center and the
neighbors in a radial layout (Figure~\ref{fig:ego-network}).
% Recall that edges are only shown between adjacent bands; we add visual
% indications to nodes with edges that are currently omitted due to the
% band layout on the screen.
% For example, an ellipsis can be added to a node to convey that
% additional edges exist (Figure~\ref{fig:ellipsis}).
% A color glyph, which fills up from empty to full depending on some
% variable, is added to the node background
% (top~vs.~bottom in Figure~\ref{fig:ellipsis}).
% This glyph can for example be used to convey the variable currently
% being used to order the nodes, such as degree, betweenness, or
% closeness centrality.


\begin{figure}[tbh]
  \centering
  % \subfigure[Hidden edges and glyph.]{
  %   \resizebox{0.19\textwidth}{!}{\includegraphics{figures/ellipsis}}
  %   \label{fig:ellipsis}
  % }
  \subfigure[Word cloud of node contents.]{
    \resizebox{0.45\textwidth}{!}{\includegraphics{figures/word-cloud}}
    \label{fig:wordcloud}
  }
  \subfigure[An ego-network popup]{
    \resizebox{0.27\textwidth}{!}{\includegraphics{figures/ego-network}}
    \label{fig:ego-network}
  }
  \caption{Exploring additional visual representations for node
    attributes.}
  \label{fig:node-attributes}
\end{figure}

\paragraph{Edge Representation.}

The display space between adjacent bands is reserved for rendering the
between-mode edges that span the neighboring node bands.
We use a edge representation based on graphical lines connecting the
two closest parts of the nodes in adjacent bands.
For within-mode networks, i.e., those that connect nodes within the
same band, several design alternatives exist.
One approach is to use arcs---similar to arc
diagrams~\cite{Wattenberg2002} and MatLink~\cite{Henry2007}---since
the nodes reside in the same band.
However, we found that these approaches make a narrow vertical band too
cluttered.
Another alternative is to provide an ego-network that only shows the
within-mode network of a particular node (Figure~\ref{fig:within-mode}),
which we implemented in MMGraph as the  ``within-network'' view.
The within-network view is slightly different from the ego-network
popup, which shows all the neighboring nodes regardless of bands that it
belongs to without showing any edges.
Though the within-network view only shows neighboring nodes in the
within-mode network, the between-mode edges between the neighboring nodes
and the nodes in the adjacent bands are also shown.
This could provide additional insights, such as ``how are my
collaborators associated with other projects and institutes?'' as shown
in Figure~\ref{fig:pi_withinmode}.

\begin{figure}[b]
  \centering
  \resizebox{\columnwidth}{!}{\includegraphics{figures/within-mode}}
  \caption{The within-network view}
  \label{fig:within-mode}
\end{figure}

% \paragraph{Navigation.}

% By separating nodes based on mode, the bands can now be zoomed and
% panned independently (Figure~\ref{fig:navigation}) while maintaining
% the visual links between them (these will change to accommodate the
% bands moving in relation to each other).
% Panning is similar to spinning the reels in a slot machine, and allows
% the user to line up different parts of adjacent bands side-by-side.
% Zooming is restricted to the vertical axis and changes the number of
% stacked nodes that are visible.
% In addition, two or more bands can be locked together so that they are
% zoomed and panned in unison instead of independently.

% \begin{figure}[t]
%   \centering
%   \resizebox{0.8\columnwidth}{!}{\includegraphics{figures/navigation}}
%   \caption{Panning and zooming in PNLB representations.}
%   \label{fig:navigation}
% \end{figure}

\paragraph{Within-Mode Sorting.}

Organizing nodes within a particular band is akin to graph layout on a
single graphical axis (the vertical axis).
The order of nodes inside the stack for each band is a free parameter,
and can be controlled in several different ways (exposed to the user):

\vspace{-.25cm}
\begin{itemize}
  \setlength{\itemsep}{0px}
\item\textbf{Node attributes:} The user may sort nodes based on node
  attributes such as name, age, or income.

\item\textbf{Edge attributes:} Edge attributes, such as time or
  weight, or the number of edges to a particular node (i.e., its total
  degree, or its degree within a specific mode) can be used for
  sorting.
  
\item\textbf{Connectivity:} A common ordering is to reorder a band by
  its connection to nodes in another band.
  We support this by a ``bring-to-top'' command that reorders all
  bands to bring all the neighbors of a particular node to the top of
  their bands.

\end{itemize}
\vspace{-.25cm}

Out of these three approaches, sorting by edge attributes, particularly
by between-mode centrality (implemented as ``connection to the right''
or ``connection to the left'' in the system) could be relevant to mSNA.
These features can easily answer questions such as ``Who is the most
successful grant writer? (Q1a, q1),'' because it is equivalent to
finding ``Which node in the PI mode has the highest degree centrality
based on its ties to the projects mode.''

\paragraph{Open Sesame.} 

To close the loop between PNLBs and parallel coordinate
plots, we propose a band extension mode where a selected band is
parted in half using an animation by invoking what we call an
``open-sesame'' interaction.
The parting animation then exposes a parallel coordinate display for
multivariate data in the space between the labels.
This display would be used to represent multivariate node data, such
as time stamps, quantities, and ordinal values.
Figure~\ref{fig:pnlbs-example} shows a screenshot from our implementation
where the mode ``Projects'' has been expanded in this way.
This interesting combination of PNLBs and parallel coordinates is
matched with Q1c.
This approach is also more powerful than other encoding approaches
because it can present multiple attributes of nodes in a mode at the
same time, and it can be used in conjunction with existing interaction
techniques developed for parallel coordinates.
For example, filtering over multiple dimensions (e.g., time and grant
amounts in Figure~\ref{fig:pnlbs-example}) turned out to be a powerful
way to select a set of nodes and edges out of the global network.

\paragraph{Additional Interaction and Navigation.}

Beyond the above interactions, our iterative design process caused us
to add several additional interaction and navigation techniques to the
MMGraph tool: highlighting (by hovering over a node), brushing (by
selecting one or several nodes), searching (by entering queries in a
text box), panning, and zooming (inspired by
TableLens~\cite{Rao1994} as shown in Figure~\ref{fig:table-lens}).

\begin{figure}[htb]
  \centering
  \resizebox{\columnwidth}{!}{\includegraphics{figures/table-lens}}
  \caption{Data abstraction mode inspired by the TableLens~\cite{Rao1994}.}
  \label{fig:table-lens}
\end{figure}
\vspace{-.25cm}
%% ---------------------------------------------------------------------
%% PHASE III: USER STUDY
%% ---------------------------------------------------------------------
\section{Phase III: Formative Qualitative Evaluation}
\label{sec:formative-eval}

As part our iterative design process, we conducted a qualitative study
to evaluate our prototype implementation for mSNA tasks.

\subsection{Method}

We invited five domain experts (4 graduate students and 1 faculty
member from our university's School of Communication) as study
participants.
All participants had been professionally analyzing social networks for
more than one year (mean = 2.7 years).
They reported that they had experience in network data, such as
Facebook friends networks, terrorist networks, donation networks, and
authorship networks.
All of them had experience with the SNA tool UCINET~\cite{Borgatti2002}.

An experimenter, the second author of this paper who was not involved
directly in developing the system, administered the experiments for all
five participants.
At the beginning of each experiment, the experimenter described the
experimental procedure and tool for ten minutes.
For the next forty minutes, each participant was asked to use the tool
to answer four mSNA tasks which are explained in
Section~\ref{sec:exp_1_tasks}.
During the task, each participant was encouraged to think aloud.
The experimenter also engaged participants in conversation by asking
questions.
After a participant completed all the tasks, the experimenter
interviewed the participant about the experiment and the tool for ten
minutes.
We collected audio-recorded conversation and screen activities of the
experiments using Camtasia.
Each experiment lasted around one hour.

\subsection{Dataset}
\label{sec:dataset}

Because of its general interest to scientists in the United States, we
decided to use a multimodal dataset derived from funding data from the
U.S.\ National Science Foundation (NSF).
NSF provides a publicly available database of awarded grants that
dates back several decades.
The NSF award search\footnote{\url{http://www.nsf.gov/awardsearch/}}
allows for advanced queries and saving search results as tabular data.
Using this tabular data as source, we built a multimodal graph using a
process similar to Liu et al.~\cite{Liu2011}, and stored it as GraphML
with a node attribute encoding the mode.

Since the entire NSF funding dataset is large in size (some 330,000
awards), we narrowed it down by specifying the awardee organization as
Purdue University and awarded years as 2003 to 2012.
Despite only selecting Purdue, the dataset includes a total of 95
institutions because many projects have external partners.
Our final dataset includes six modes: 315 projects, 205 programs, 507
PIs and co-PIs (PIs henceforth), 95 institutions, 160 program managers
(Pr-Man), and 9 directorates (henceforth Dir).
We merged PIs and Co-PIs into a single mode (PI) to avoid duplicating
individuals.
This also yielded a within-mode network that connected investigators
with their collaborators.

Figure~\ref{fig:sixmodes} shows our entity-relationship diagram for
the dataset.
A PI is affiliated with one or more institutions.
A PI collaborates with zero or more PIs (this is the only within-mode
network in this NSF dataset).
A PI is involved in one or more projects.
Each project is awarded by one program manager (Pr-Man).
Each project belongs to one or more programs.
A program manager belongs to one or more programs.
A program belongs to a directorate (Dir).

\begin{figure}[htb]
  \centering
  \resizebox{\columnwidth}{!}{\includegraphics{figures/sixmodes2}}
  \caption{The entity-relationship diagram of six modes of the NSF
    funding award data used in the experiment.}
  \label{fig:sixmodes}
\end{figure}
\vspace{-.25cm}

\subsection{Tasks}
\label{sec:exp_1_tasks}

The tasks in the experiment included the following questions:

\vspace{-.25cm}
\begin{itemize}
  \setlength{\itemsep}{0px}
\item[T1] Who is the most successful PI in terms of number of awards?
\item[T2] Are there PIs who have been awarded several grants together?
\item[T3] Do some program managers often award grants to the same PIs?
\item[T4] Which are projects that have been awarded more than \$70M,
  and what are their commonalities?
\end{itemize}
\vspace{-.25cm}

While some of these tasks could certainly be solved using
computational means, perhaps even more efficiently than using visual
analytics, we wanted to include a broad spectrum of tasks to reflect
how an mSNA tool is used in realistic settings.
Even though we are strong believers in visual analytics augmenting
existing (computational) tools, it is also true that switching to
another application in mid-analysis may break the user's flow.
For this reason, our tasks were motivated by the graph task taxonomy
proposed by Lee et al~\cite{Lee2006}.
Thus, task T1 could be answered by analyzing the between-mode network
between PI and Project, whereas T2 required analyzing the within-mode
network of the PI mode.
To answer T3, participants would have to connect the indirect
relationship between Program Manager and PI.
Finally, T4 was an open-ended task that required participants to delve
into network data by formulating their own hypotheses and testing them
using the visual analytics tool.
Based on our argument above, we started from basic tasks (T1 and T2)
that could be answered with a simple interaction, and gradually
exposed participants to more difficult tasks.
We also let participants pursue any interesting serendipitous
observations found throughout the analysis process and report on their
findings.

\begin{figure*}[t!]
  \centering
  \subfigure[PIs sorted by projects.]{
    \fbox{%
      \resizebox{0.25\textwidth}{!}{
        \includegraphics{figures/pi_connections}}%
    }
    \label{fig:pi_connections}
  }
  \subfigure[The within-network of a PI node.]{
    \fbox{%
      \resizebox{0.293\textwidth}{!}{
        \includegraphics{figures/pi_withinmode}}%
    }
    \label{fig:pi_withinmode}
  }
  \subfigure[Open-sesame for project amounts.]{
    \fbox{%
      \resizebox{0.345\textwidth}{!}{
        \includegraphics{figures/projects_opensesame}}%
    }
    \label{fig:projects_opensesame}
  }
  \caption{Screenshots of the PNLBs view highlighting several user
    interactions during the user study.}
  \label{fig:user_screenshots}
\end{figure*}

\subsection{Results}
In general, participants successfully completed all tasks.
T1 and T2 were completed without major problems, and using similar
approaches.
To complete T1, participants sorted the PIs mode by connection to the
Projects mode (the degree centrality of the between-mode network
between the PI and Project modes), which shows that Kevin Webb (a
professor of electrical and computer engineering) has seven projects
(Figure~\ref{fig:pi_connections}).
T2 does not have a definite answer, but participants found that they
could sort the PIs mode by within-mode connections (sort by the degree
within a specific mode) to find PIs who have high collaborations with
others and check whether some of them were repeated collaborators
using the within-network view (Figure~\ref{fig:pi_withinmode}).

In contrast, participants' approaches varied when solving T3.
Some used brushing to see the relationship between PIs, Projects, and
Program Manager to see if there were any common occurrences over
multiple projects.
Others simply used ego-networks to see if the same pair of PI and
program manager appeared at the same time.

Since T4 is a more open-ended task, the usages of PNLBs were more
diverse.
Initially, all the participants successfully used the open-sesame
interaction to find the project awarded more than \$70M (see
Figure~\ref{fig:projects_opensesame}).
Subsequently, some used word clouds over projects; others moved all of
the associated PIs and program managers to the top of the band to see
their relationships.
Some even investigated beyond the specific scope of T4 and searched
for what other projects the PIs of the 70 million-dollar project have
worked on and explored that data.

All participants stated that they preferred the PNLBs view over the
compound network visualization for virtually all mSNA tasks.
In particular, they liked the idea of ``dissecting complicated
datasets into multiple bimodal sets,'' so that they could focus
on two adjacent bands at a time.
They noted that this tool was easy to use for first-time users, and
that it took less than an hour to be proficient at using it.
One participant also stated that ``patterns and insights found in this
view are more digestible due to its structure [than the compound network visualization].''
However, several participants also stressed that having the mutually
interlinked compound network visualization and the PNLBs view allowed
for transferring details from multimodal networks to the network
overview and vice versa.

Participants also felt that some functions in the PNLBs view were
particularly helpful for mSNA tasks.
For example, they all seemed to enjoy how interactions often brought
about new hypotheses and easily supported exploring them.
One participant used the open-sesame interaction to determine the
project with the largest grant amount.
She then used the word cloud to view the details of that project.
This made her curious if other projects with similar keywords had been
awarded.
She queried the keyword and found another project, and viewed the
ego-network of that project.
Several PIs were common to both projects, and the participant then
viewed their within-network.
She wondered if these projects were granted by the same program
manager, and so brought all their program managers within view.
Such long sequences of actions were well-supported by our tool,
something which the participants informed us that they appreciated.

%% ---------------------------------------------------------------------
%% PHASE IV: Iterative MMGraph Refinement
%% ---------------------------------------------------------------------
\section{Phase IV: Iterative MMGraph Refinement}

The purpose of our formative evaluation was to evoke feedback on the
\textsc{MMGraph} tool, which to that point had only been guided by A3,
our social scientist co-author.
Below we review this feedback and then discuss the concrete changes we
made to MMGraph in response.
 
\subsection{Formative Feedback}

All participants expressed a desire to be able to use the tool with
their own social networks, such as authorship networks, donation
networks, terrorist networks, and organizational networks.
One participant, who has an interest in terrorist networks, believed
that the tool could be helpful for finding patterns in terrorist
networks by studying the relationship between sponsors, operators, and
cell leaders.
Another participant, who studies donation networks, stated that the
tool could be used to identify whom she should target for raising
certain types of donations by analyzing previous interactions.
She hoped to use open-sesame interaction to view the key
characteristics of potential donors.

Some weaknesses of the PNLBs view were also uncovered.
Some participants disliked the fact that the compound network
visualization was underused, and some wanted to use the view as a canvas
where they could filter out the nodes they did not want to study.
Others proposed clustering the nodes in the compound network visualization based on
the projected network or other network metrics.
Most participants wanted to be able to see connections beyond the most
adjacent modes in the PNLBs view.
They wanted to have an ego-network starting from a node in one band to
all nodes in all bands, which would provide a good summary for a
particular node of interest.
They also proposed streamlining the interface by replacing menus with
toolbars and dialog boxes to make the exploration faster, more
effortless, and more discoverable.

Perhaps the most significant feedback, echoed by several participants,
was requests for integrating traditional network metrics into the
tool.
In other words, if MMGraph to that point had emphasized visual aspects
of multimodal graphs, this feedback effectively highlighted the need
for computational metrics and analytics components in the tool.

\subsection{MMGraph Refinements}
\label{sub:mmgraph_refinements}

We used much of the participant feedback to make incremental
improvements to MMGraph, including several user interface and
interaction refinements.
Participants also noted that the node representation could be utilized
to visualize particular network and data attributes.
We implemented this new feature as the color glyphs
(Figure~\ref{fig:ellipsis}).

\begin{figure}[tbh]
  \centering
  \resizebox{0.19\textwidth}{!}{\includegraphics{figures/ellipsis-without-glyph}}
  \caption{A color glyph on a node}
  \label{fig:ellipsis}
\end{figure}

Beyond such minor changes, the most significant feature we added to
the MMGraph tool during this second development phase was the ability
to order, filter, and visualize nodes based on additional network
metrics.
Of course, the fact that most of these metrics are defined for
unimodal and not multimodal networks meant that we had to create these
definitions ourselves:

\vspace{-.25cm}
\begin{itemize}
  \setlength{\itemsep}{0px}

\item\textbf{Multimodal degree centrality:} the number of edges for a
  vertex that connect other vertices either within the same mode
  (\textit{within-mode degree centrality}), or to vertices only in
  other modes (\textit{between-mode degree centrality}).
  Can also be defined for a specific mode, i.e., the degree centrality
  for a vertex to mode A.

  This is a measure for how well-connected a vertex is to vertices in
  one or more modes in the multimodal network.

\item\textbf{Multimodal betweenness centrality:} the multimodal
  betweenness centrality of a vertex in one mode is equal to the
  number of shortest paths from all vertices to all others in other
  modes that pass through that vertex.
  Can also be defined for specific modes.

  The intuition of this measure is that it captures how important the
  vertex is in connecting vertices in other modes to each other.
  For example, in a co-authorship network, papers with high multimodal
  betweenness centrality connects many authors who otherwise have not
  collaborated.

\item\textbf{Multimodal closeness centrality:} the sum of the shortest
  paths from a certain vertex in one mode to all the other vertices in
  other modes.
  Again, can be similarly defined for specific modes.

  This measure should be interpreted as the distance of the vertex to
  all other vertices in their modes.
  For example, in a database-user network, a database with low
  closeness centrality would quickly disseminate information to all
  users.

\end{itemize}
\vspace{-.25cm}

%% ---------------------------------------------------------------------
%% PHASE V: Summative Qualitative Evaluation
%% ---------------------------------------------------------------------
\section{Phase V: Summative Qualitative Evaluation}

Finally, to validate our improvements and new analytics capabilities
in MMGraph, we conducted a second qualitative evaluation.
We used the same NSF dataset as in the previous evaluation
(Section~\ref{sec:dataset}).

\subsection{Method}

We recruited three domain experts: one faculty member (P1) and two
graduate students (P2 and P3) from the same school of Communication 
to learn how our improved MMGraph supported mSNA.
P1 had also participated in the previous formative qualitative
evaluation, but P2 and P3 had never seen MMGraph before.
Instead of providing pre-created tasks as in
Section~\ref{sec:exp_1_tasks}, in this phase, we allowed participants to
freely study the NSF network, generate questions, and solve
them using MMGraph.
The purpose for this phase was to observe how the newly added
features (the node attribute visualization in
Figure~\ref{fig:ellipsis} and additional multimodal network metrics)
were used and, more importantly, what kinds of other tasks 
participants would like to perform.

\subsection{Results}

Overall, all participants successfully understood how to use the tool,
did not encounter major problems, and were positive about its
capabilities.
The open-ended evaluation design worked well; all participants used
the tool to come up with interesting network questions.
In fact, this allowed us to both validate MMGraph's new features
introduced in Phase IV as well as continue to study its general
utility for mSNA.

\subsubsection{Validation of New Features}

The newly added multimodal network metrics (Section~\ref{sub:mmgraph_refinements}) were appreciated by our participants.
For example, P2 wanted to determine the researcher at
Purdue University who had been awarded the most NSF grants with PIs
from other institutions.
The participant found the answer by sorting the PI band by multimodal
betweenness centrality.
Similarly, P3 wanted to find program managers who provide
grants to the largest number of PIs, or, in other words, program
manager who accepted a variety of different PIs.
Again, sorting program managers by multimodal betweenness centrality
with respect to PI yielded this information.
A participant noted that sorting a band by centrality measures can
show how closeness, betweenness, or degree can impact the tie density
to other bands using MMGraph.

The fact that MMGraph now also encodes other node attributes (such
as funding dates) using color glyphs also helped participants quickly
make sense of the dataset and generate interesting questions to explore
further.
Earlier in the session, P1 found that a
certain rotating program manager had awarded grants to more than 10
different projects in the participant's own research area.
This was an entirely new insight for P1, and the
participant was curious about the availability of the program manager
since such rotating program managers tend to stay no longer than a few
years.
Surveying the node attributes, the participant performed the open-sesame
interaction to see which dates the grants had been awarded by the
particular program manager.
It turned out that the program manager had granted all those projects
in 2008 and 2009.
This led the participant to conclude that the program manager had
finished his or her tenure at the NSF, and was likely not a good
contact for future grant proposals.

\subsubsection{Impacts of MMGraph on mSNA}

Beyond validating the new features added to MMGraph, we were
also able to more clearly observe how MMGraph impacts mSNA processes.
Our participants articulated the impacts more clearly in Phase V 
probably because we did not give specific tasks to participants in this phase.
Particularly, we had an interesting discussion with P1,
who mentioned that a social network analyst often starts his or her investigation 
with a particular node instead of the whole network.
The node is selected based on various network metrics that analysts
can calculate, such as centrality.
After learning about the node, the analyst gradually expands the scope
of investigation to connected nodes.

In contrast, in our evaluation sessions, participants demonstrated that they 
conducted a more structured, step-by-step investigation as follows:
First, using PNLBs, participants were able to understand
the overview of the between-mode relationships across all modes.
The overview led participants to successfully exclude uninteresting between-mode network (e.g., having a low number of edges drawn between the two mode).
Second, after the overview, participants investigated the mode-to-mode
relationships instead of delving into a single node. 
Using multimodal social network metrics, participants were able to sort a band
of interest and understand the correlation between the band with others.
Participants were also able to easily investigate the
correlation between the within-mode network of one mode and the
between-mode network with another mode.
Third, after finding an interesting phenomenon between modes, participants
investigated details of interesting nodes by using the open-sesame interaction
and/or the popup view for word clouds.
The three steps iteratively repeated throughout mSNA processes. 

%% ---------------------------------------------------------------------
%% DISCUSSION
%% ---------------------------------------------------------------------
\section{Discussion}

Our two qualitative evaluations raised many interesting points and
insights; we highlight the important ones below.

\subsection{Benefits of PNLBs on mSNA}

Throughout the design study, we learned that MMGraph provides several benefits over common mSNA approaches (e.g., the compound network visualization and network metrics).

First, MMGraph, especially PNLBs, provides an effective structure for mSNA, which cannot be supported by the compound network visualization.
We believe that the compound network visualization itself contains too much information in a single view without a proper abstraction.
This complexity hampers social scientists from seeing the big picture, and may lead them to focus on an individual node at the outset of investigation.
In contrast, the mode-by-mode division provided by PNLBs seems to be a proper external representation that social scientists can easily understand and work with.
The zoomed-out view of PNLBs worked as a useful overview of the whole multimodal network data though it does not readily provide within-mode ties. 
Participants also effectively focused on mode-to-mode relationships.
This finding is consistent with the lesson in our previous study~\cite{kwon_evaluating_2012}, where an explicit visualization of temporal data changes the investigative analysis process to become more of a top-down process, rather than a bottom-up process.

Second, a tight integration between network metrics and visual representation provides a seamless train of analysis. 
As shown in the two evaluation studies, participants appreciated that 
various network metrics are used for ordering and encoding through glyphs instead of being simply presented as a list of numbers.
We believe that this tight integration allowed our participants to iteratively ask questions and answer them immediately, using that insight in their next question.
Such train of analysis could be easily broken if one has to export data so that he or she could run statistical tests in a separate program.

\subsection{Divide-and-Conquer and Details-On-Demand}

The design strategy used for PNLBs is based on the divide-and-conquer approach (see Section~\ref{sec:dimensionality}) as well as showing relations on demand.
We intended to minimize visual clutter and complexity, yet it forced users to perform mSNA tasks more efficiently.

We chose the between-mode network as the dominant visualization for PNLBs because our investigation revealed that between-mode networks tended to be most interesting to social scientists.
During the evaluation sessions (Phase III and V), we learned that research questions are often related to finding nodes in a mode that play a central role in certain events (e.g., donations, terrorist plot, and idea generation), which could be answered by dissecting the multimodal network into bimodal networks and analyzing them separately.

In addition, to satisfy the needs for further investigation, we designed the PNLBs view to show the within-mode network and other node attribute information \emph{on demand}.
Since representations for the between-mode ties and bands already occupy the whole display, we devised an option to let the user view the within-mode network of each node as the within-network view (Figure~\ref{fig:within-mode}).
In addition, while using the within-mode network view, users could still
preserve the overview of the compound network visualization and PNLBs.
Beyond the within-network popup, we provide multivariate node attributes using a dynamically activated parallel coordinate view
using the open-sesame interaction only at the explicit request of the user
(Figure~\ref{fig:projects_opensesame}).

Some may argue that such a discontinuous analysis path may harm users
because users cannot see the overall picture all at once.
However, in our evaluations, we found that this step-by-step approach
seemed well-suited to how our users performed the sensemaking task.
As participants revealed in their comments, the compound network visualization easily becomes too busy when the number of nodes and edges increase.
The apparent lack of structure does not provide useful cues to guide
the analysis.
PNLBs, on the other hand, force users to view networks
partitioned into modes by revealing only a limited amount of
connectivity information at a time.
Starting from this very structured representation, users can then
progressively unlock additional information on-demand using
interaction.
Each interaction uncovers an additional piece of information, enabling
a progressive refinement of the analysis~\cite{Gotz2006}.
In other words, PNLBs may provide less information than the compound network visualization in a single view, but appear to encourage structured exploration.

\subsection{mSNA Tasks}

Throughout the design study, we gradually learned more about what kinds of tasks social scientists would like to perform during mSNA.
Many tasks came from observations of how participants explored and learned about the provided NSF data using MMGraph.
Other tasks were captured from the discussion regarding their own data and how they would analyze their data using MMGraph.
While the following list is not exhaustive, it could be an initial set of tasks for mSNA, which may contribute to increase task clarity in mSNA.
Note that we did not include any common tasks that also can be found in unimodal SNA (e.g., seeing the distribution of a network metric over a particular mode), and the design of MMGraph could bias the kinds of tasks that we elicited below:

\vspace{-.25cm}
\begin{itemize}
\setlength{\itemsep}{-1mm}
\item Identify a node with an extreme network metric w.r.t.\ another
      mode. (e.g., 
      In media network with media, words, and readers as modes,
      P3 wanted to find the most central medium, word, and reader
      with respect to each other.)
\item Correlate within-mode and between-mode network.
      (e.g., In the NSF dataset, participants were curious whether
      a successful collaboration (within-mode network) repeated
      on other projects (between-mode network).)
\item Correlate attributes and between-mode network.
      (e.g., In Wikipedia edit network with authors, edits, and discussions as modes, P2 wanted to view correlation between the edit date (attribute)
      and the edit counts from a group of authors (between-mode network).)
\item Correlate attributes and within-mode network.
      (e.g., In university student network, P2 wanted to view correlation of
      cultural background of students (attribute) with their friendships (within-mode network).)
\end{itemize}
\vspace{-.25cm}

Together with unimodal graph tasks~\cite{Lee2006}, these are the types
of tasks that visual analytics tools for mSNA should support.

\subsection{Visual Analytics Studies with Social Scientists}
\label{sec:difficulties_of_socialscientists}

We here discuss problems that our social science collaborators
encountered while working with visualization researchers on this
project.
By their nature, visualization researchers tend to be technique-driven
and are eager to come up with novel visualizations.
This makes it difficult for domain experts to emphasize the problems
to be solved. 
We found that some ideas originating from our social scientist
collaborators tended to be discouraged or downplayed simply because
they were not novel enough or too complex to implement.
Furthermore, the analysis of social network data in social science
fields is usually driven by established theories.
In this sense, the exploratory analysis of data
supported by visual analytics yielded a radically different approach
to sensemaking than traditional methods, which caused some
difficulties to bridge.
Lastly, initial paper prototypes may not be sufficient for social
scientists to fully understand the significance and potential of a
technique.
We found that our social scientist colleagues were best able to
understand when confronted by a concrete visualization (e.g., PNLBs)
with a specific context (e.g., the NSF multimodal dataset) as Lloyd and Dykes~\cite{lloyd_dykes_2011} suggest.

%% ---------------------------------------------------------------------
%% CONCLUSION
%% ---------------------------------------------------------------------
\section{Conclusions and Future Work}

We have presented a design study on applying visual analytics to
multimodal social network analysis (mSNA).
Our study, which involves a social scientist at the outset and was 
methodically conducted in five phases that each
fed the next, involved initial early design, iterative tool
development, a formative evaluation, a second iterative development
phase, and a final summative evaluation.
The resulting MMGraph visual analytics tool combines both a compound
network visualization as well as a view using a visualization technique
called parallel node-link bands (PNLBs).
Through the iterative design processes, we not only refined MMGraph but
also learned what kinds of tasks social scientists would like to conduct
while analyzing a multimodal network.
We also learned the effectiveness of the divide-and-conquer and
details-on-demand approaches in designing an effective tool for
social scientists.

The visual analytics tools and techniques designed in this design
study are only a small group of methods specifically designed for such
tasks, and we think that the space is wide open for further work.
For example, multimodal networks often represent affiliation ties and
social circles, which provide conditions for future connections.
In this sense, visual analytics of longitudinal multimodal networks
can expand our understanding of network dynamics.
Although we implemented time as a feature in the open-sesame
interaction to explore some basic questions (not evaluated in the
study), continued work is needed to answer more complex questions.

\section*{Acknowledgments}

We thank our social scientist colleagues who participated in the study
as well as Sukwon Lee, Sung-Hee Kim, and Aditya Srinath who provided constructive feedback on this study.
This work was partly supported by the U.S.\ Department of Homeland
Security's VACCINE Center under award 2009-ST-061-CI0003.

% --------------------------------------------------------------------
% Bibliography
% --------------------------------------------------------------------
\bibliographystyle{abbrv}
\bibliography{mmgraph}

\end{document}